# -*-coding:utf-8-*- 
# @Time : 2023/7/22 10:10
# @Author : Li
# @File : run.py
import json

from datetime import datetime
from scrapy import cmdline, spiderloader
from scrapy.crawler import CrawlerProcess
from Knowledge_Planet.untils import get_cookie
from scrapy.utils.project import get_project_settings



# if __name__ == '__main__':
#     cmdline.execute('scrapy crawl selen'.split())
def run_spider():
    settings = get_project_settings()
    process = CrawlerProcess(settings)
    # 查看现有爬虫
    spider_loader = spiderloader.SpiderLoader.from_settings(settings)
    print(spider_loader.list())
    # 将 spider 逐个添加到 CrawlerProcess 实例及 crawlers 列表中
    crawlers = []
    for spider in spider_loader.list():
        print(f'Running spider {spider}')
        crawler = process.create_crawler(spider)
        crawlers.append(crawler)
        # process.crawl(crawler, cookie, end_time)
        process.crawl(crawler, cookie, end_time)
    # 开始爬虫
    process.start()
    # 获取爬虫的统计信息
    stats_dict = {}
    for crawler in crawlers:
        stats_dict[crawler.spider.name] = crawler.stats.get_stats()
    return stats_dict


def main():
    spider_status = run_spider()
    print(spider_status)


if __name__ == '__main__':
    # 获取cookie存储到本地
    get_cookie.get_cookie()  # 扫码登录
    with open("cookie.txt", "r", encoding="utf-8") as f1:
        cookie = json.loads(f1.readlines()[0])
    # 每次启动截取时间, 输入的日期时间字符串
    end_time = ""  # 截止时间  格式：2023-04-08 15:16:54
    if end_time:
        # 将日期时间字符串解析为datetime对象，指定输入的日期时间格式
        input_format = '%Y-%m-%d %H:%M:%S'
        datetime_obj = datetime.strptime(end_time, input_format)
        # 格式化为iso格式（包含时区信息），并保留三位小数
        formatted_datetime = datetime_obj.isoformat(timespec='milliseconds')
        # 输出转换后的日期时间字符串
        end_time = formatted_datetime + str("+0800")
    # print(cookie,end_time)
    # 执行主程序
    main()
